package com.googlecode.gaal.preprocess.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.googlecode.gaal.preprocess.api.Normalizer;
import com.googlecode.gaal.preprocess.api.Tokenizer;

/**
 * Tokenize while using the new line as a document boundary
 * 
 * @author akislev
 * 
 */
public class MultidocumentRegexTokenizer implements Tokenizer<String> {
    private final BufferedReader reader;
    private final Normalizer<String> normalizer;
    private final Matcher matcher;
    private int documentId;

    public MultidocumentRegexTokenizer(Reader reader, String pattern, Normalizer<String> normalizer) {
        this(reader, Pattern.compile(pattern, Pattern.COMMENTS), normalizer);
    }

    public MultidocumentRegexTokenizer(Reader reader, Pattern pattern, final Normalizer<String> normalizer) {
        this.reader = new BufferedReader(reader);
        this.matcher = pattern.matcher("");
        this.normalizer = normalizer;
    }

    @Override
    public Iterator<Document<String>> iterator() {
        return new DocumentIterator();

    }

    private class DocumentIterator implements Iterator<Document<String>> {
        private Document<String> next = nextDocument();

        @Override
        public boolean hasNext() {
            return next != null;
        }

        @Override
        public Document<String> next() {
            Document<String> document = next;
            next = nextDocument();
            return document;
        }

        private Document<String> nextDocument() {
            try {
                String line = reader.readLine();
                if (line == null) {
                    return null;
                }
                return new DocumentImpl(documentId++, line);

            } catch (IOException e) {
                throw new IllegalStateException(e);
            }
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    }

    private class DocumentImpl implements Document<String> {
        private final int id;
        private final CharSequence line;

        public DocumentImpl(int id, CharSequence line) {
            this.id = id;
            this.line = line;
        }

        @Override
        public Iterator<String> iterator() {
            matcher.reset(line);
            return new Iterator<String>() {
                String next = nextToken();

                @Override
                public boolean hasNext() throws IllegalStateException {
                    return next != null;
                }

                @Override
                public String next() {
                    String token = next;
                    next = nextToken();
                    return token;
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }

        private String nextToken() {
            while (matcher.find()) {
                String token = normalizer.normalize(matcher.group());
                if (token != null)
                    return token;
            }
            return null;
        }

        @Override
        public int getId() {
            return id;
        }
    }
}
